package net.lateeye.search;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
import java.util.LinkedList;
import java.util.Properties;
import java.util.Random;

import net.lateeye.util.MediaProperties;
import net.lateeye.util.NoValueSpecifiedException;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

/**
 * Extends java.sql.Statement interface.
 * 
 * @author Isao Isaac Saito <130s@lateeye.net>
 * @since 2.0
 * 
 */
public abstract class AbsSearchEngineClient implements Statement
{
	protected static final String CHARSET_UTF = "UTF-8";

	protected MediaProperties props = null;
	// Default value. Can be read from properties file.
	private int max_interval = 15; // 1/11/11) This default value will never be
	// used.
	/**
	 * Minimum smalles query interval (in second). Purpose is to avoid accessing
	 * external search services so often.
	 * 
	 * @author Isao Isaac Saito <130s@lateeye.net>
	 * @since 1/11/11
	 */
	private static final int MIN_SMALLEST_QUERY_INTERVAL = 3;
	private Random rand = new Random();
	/**
	 * To be modified when the class is instantiated by constructor 120 sec(?)
	 * as default. TODO Needs to be figured out if its unit is second or else.
	 * 
	 * @since 10Dec30
	 */
	private int queryTimeout = 120;

	/**
	 * Accessed by setter/getter fetchSize(). Default is 1 million, which seems
	 * to be default on google.com.
	 * */
	private int fetchSize = 1000000;

	/**
	 * 
	 * @return java.sql.ResultSet Needs to be casted to AbsResultSet when in
	 *         use.
	 * @throws SQLException
	 *             Encapulate a returned exception from doGetResultSet(String,
	 *             int, String, AbsSearchResultPageTextProcessor)
	 * @see java.sql.Statement#executeQuery(java.lang.String)
	 */
	public ResultSet executeQuery(String query) throws SQLException
	{
		AbsResultSet rs = null;
		int num = Integer.parseInt(this.props
				.getProperty(SearchengineProps.PROPKEY_VAL_NUM_SEARCHRESULTS));
		try {
			rs = this.doGetResultSet(query, num, this
					.buildDefaultSearchConditions(), this.getAnalyzer());
		} catch (IOException e) {
			throw new SQLException(e.getMessage());
		}
		return rs;
	}

	/**
	 * @since 2.0
	 * @return
	 */
	protected abstract AbsSearchResultPageTextProcessor getAnalyzer();

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#addBatch(java.lang.String)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void addBatch(String sql) throws SQLException
	{
		// TODO Auto-generated method stub

	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#cancel()
	 * 
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void cancel() throws SQLException
	{
		// TODO Auto-generated method stub
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#clearBatch()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void clearBatch() throws SQLException
	{
		// TODO Auto-generated method stub

	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#clearWarnings()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void clearWarnings() throws SQLException
	{
		// TODO Auto-generated method stub

	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#close()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void close() throws SQLException
	{
	}

	/**
	 * Not implemented. Never to be used.
	 * 
	 * @see java.sql.Statement#execute(java.lang.String, int)
	 */
	public boolean execute(String sql, int autoGeneratedKeys)
			throws SQLException
	{
		return false;
	}

	/**
	 * Not implemented. Never to be used.
	 * 
	 * @see java.sql.Statement#execute(java.lang.String, int[])
	 */
	public boolean execute(String sql, int[] columnIndexes) throws SQLException
	{
		return false;
	}

	/**
	 * Not implemented. Never to be used.
	 * 
	 * @see java.sql.Statement#execute(java.lang.String, java.lang.String[])
	 */
	public boolean execute(String sql, String[] columnNames)
			throws SQLException
	{
		return false;
	}

	/**
	 * Not implemented. Never to be used.
	 * 
	 * @see java.sql.Statement#execute(java.lang.String)
	 */
	public boolean execute(String sql) throws SQLException
	{
		return false;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#executeBatch()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int[] executeBatch() throws SQLException
	{
		return null;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#executeUpdate(java.lang.String, int)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int executeUpdate(String sql, int autoGeneratedKeys)
			throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#executeUpdate(java.lang.String, int[])
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int executeUpdate(String sql, int[] columnIndexes)
			throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#executeUpdate(java.lang.String,
	 *      java.lang.String[])
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int executeUpdate(String sql, String[] columnNames)
			throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#executeUpdate(java.lang.String)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int executeUpdate(String sql) throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getConnection()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public Connection getConnection() throws SQLException
	{
		return null;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getFetchDirection()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getFetchDirection() throws SQLException
	{
		return 0;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getFetchSize()
	 */
	public int getFetchSize() throws SQLException
	{
		return this.fetchSize;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getGeneratedKeys()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public ResultSet getGeneratedKeys() throws SQLException
	{
		return null;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getMaxFieldSize()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getMaxFieldSize() throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getMaxRows()
	 * @deprecated 10Apr08/130s: Should not be used since I haven't understood
	 *             the difference between fetchSize, and currently make
	 *             fetchSize alive.
	 */
	public int getMaxRows() throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getMoreResults()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public boolean getMoreResults() throws SQLException
	{
		return false;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getMoreResults(int)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public boolean getMoreResults(int current) throws SQLException
	{
		return false;
	}

	/**
	 * Returns timeout period set by user. If no value is set, returns default.
	 * 
	 * @return default value is 120 seconds.
	 * @see java.sql.Statement#getQueryTimeout()
	 */
	public int getQueryTimeout() throws SQLException
	{
		return this.queryTimeout;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getResultSet()
	 * @deprecated 10Apr08/130s: Returns null. executeQuery should be used to
	 *             get result set.
	 */
	public ResultSet getResultSet() throws SQLException
	{
		return null;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getResultSetConcurrency()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getResultSetConcurrency() throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getResultSetHoldability()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getResultSetHoldability() throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getResultSetType()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getResultSetType() throws SQLException
	{
		return 0;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getUpdateCount()
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public int getUpdateCount() throws SQLException
	{
		return 0;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#getWarnings()
	 * 
	 * 10Apr08/130s/Just keep this as it is without implementation since I have
	 * no idea how to utilize this method now, but this may turn out useful, or
	 * not.
	 */
	public SQLWarning getWarnings() throws SQLException
	{
		// TODO Auto-generated method stub
		return null;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setCursorName(java.lang.String)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void setCursorName(String name) throws SQLException
	{

	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setEscapeProcessing(boolean)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void setEscapeProcessing(boolean enable) throws SQLException
	{
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setFetchDirection(int)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void setFetchDirection(int direction) throws SQLException
	{
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setFetchSize(int)
	 */
	public void setFetchSize(int rows) throws SQLException
	{
		this.fetchSize = rows;
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setMaxFieldSize(int)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void setMaxFieldSize(int max) throws SQLException
	{
	}

	/**
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setMaxRows(int)
	 * @deprecated 10Mar16/130s: Should not be used since this method is
	 *             originally designed for RDBMS and I haven't found how to take
	 *             advantage of it for web search engine purpose.
	 */
	public void setMaxRows(int max) throws SQLException
	{
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see java.sql.Statement#setQueryTimeout(int)
	 */
	public void setQueryTimeout(int seconds) throws SQLException
	{
		this.queryTimeout = seconds;
	}

	public AbsSearchEngineClient(SearchengineProps props)
			throws NoValueSpecifiedException, NumberFormatException
	{
		// 10Apr08/130s/ Change cast type to SearchenginePropsImpl since no
		// reason found for using SimpleMediawareProps of which covering area is
		// beyond search engine.
		this.props = (MediaProperties) props;

		// DO: Initiate search engine client by reading setting info from
		// properties.
		String v = null;
		try {
			v = this.props
					.getProperty(SearchengineProps.PROPKEY_INTERVAL_EXECUTEQUERY);
		} catch (NoValueSpecifiedException e) {
			throw e;
		}
		int interval = Integer.parseInt(v);
		if (interval < MIN_SMALLEST_QUERY_INTERVAL) {
			// Do) Values smaller than MIN_SMALLEST_QUERY_INTERVAL (including
			// negative numbers) is to be replaced by the minimum smallest.
			interval = MIN_SMALLEST_QUERY_INTERVAL;
			System.out
					.println(super.getClass().getName()
							+ " Constructor. Interval is replaced by the minimum smallest (i.e. "
							+ MIN_SMALLEST_QUERY_INTERVAL + " sec.)");
		}
		this.max_interval = interval;

		try {
			this.setQueryTimeout(Integer.parseInt(this.props
					.getProperty(SearchengineProps.PROPKEY_QUERY_TIMEOUT)));
		} catch (NoValueSpecifiedException e) {
			throw e;
		} catch (NumberFormatException e) {
			throw e;
		} catch (SQLException e) {
			// DO: SQLException never happens here (10Apr08/130s).
			e.printStackTrace();
		}
	}

	private int getRandomInterval()
	{
		return this.rand.nextInt(this.max_interval);
	}

	public String encodeQueryUrl(String queryStr)
	{
		String encodedQueryStr = null;
		try {
			encodedQueryStr = URLEncoder.encode(queryStr, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}

		return encodedQueryStr;// this.createSearchQueryUrl(encodedQueryStr);
	}

	/**
	 * this is intended only for so-called keyword search.
	 * 
	 * @param encodedQueryStr
	 * @param searchConditions
	 *            Should follow the format defined in each actual search engine.
	 *            Typical format is the set of conditions which consist of param
	 *            name and value connected with '=', and each pair of param and
	 *            value is differentiated by '?' symbol.
	 * @param requiredResultSize
	 * @param startPos
	 * @param sizeofUniverse
	 * @return String query string
	 */
	protected abstract String createSearchQueryUrl(String encodedQueryStr,
			int requiredResultSize, int startPos, String searchConditions);

	/**
	 * 
	 * @author Isao Isaac Saito <130s@lateeye.net>
	 * @param rawQueryStr
	 *            Not necesarily encoded. Following processes are in charge of
	 *            it.
	 * @return
	 * @throws ClientProtocolException
	 * @throws IOException
	 */
	public abstract String doGetHtmlFile(String rawQueryStr)
			throws ClientProtocolException, IOException;

	/**
	 * parse a single search result page and return ResultSet instance. This
	 * doesn't contain data for multiple docs.
	 * 
	 * changelog:2008mar13/130s/make abstract as return value is not always html
	 * (ex. original xml format by GSA).
	 * 
	 * @param queryStr
	 * @param analyzer
	 * @return
	 * @throws IOException
	 */
	public abstract AbsResultSet doGetResultSetForSinglePageOfSearchResult(
			String queryStr, AbsSearchResultPageTextProcessor analyzer)
			throws IOException;

	protected abstract String buildDefaultSearchConditions();

	/**
	 * @param queryStr
	 *            should not be encoded string. encoding will be done in this
	 *            method
	 * @param requiredResultSize
	 *            Size of result contents.
	 * @param searchConditions
	 *            Should follow the format defined in each actual search engine.
	 *            Typical format is the set of conditions which consist of param
	 *            name and value connected with '=', and each pair of param and
	 *            value is differentiated by '?' symbol.
	 * @param analyzer
	 *            Text (usually html) parser that creates result set by
	 *            extracting data from search result page.
	 * @return if no resultset found then null.
	 * @throws IOException
	 *             Possibly happen when connecting to search engines.
	 */
	public AbsResultSet doGetResultSet(String queryStr, int requiredResultSize,
			String searchConditions, AbsSearchResultPageTextProcessor analyzer)
			throws IOException
	{
		int repeatTimes = 1;
		int numofResultsOnSingleResultpage = Integer
				.parseInt(this.props
						.getProperty(SearchengineProps.PROPKEY_VAL_NUMOFRESULTS_SINGLEPAGE));

		if (numofResultsOnSingleResultpage < requiredResultSize) {
			// Do; determine the number of times necessary to obtain specified
			// number of results
			repeatTimes = requiredResultSize / numofResultsOnSingleResultpage;
		}

		AbsResultSet rs = null;
		long numofReturnedDoc = 0;
		// Do; repeat executing query to search engine n times which was
		// determined above by the ratio of
		// #requiredResultSize / #resultsOnSinglePage
		for (int i = 0; i < repeatTimes; i++) {
			int startPos = i * numofResultsOnSingleResultpage;
			String encodedQueryStr = this.encodeQueryUrl(queryStr);
			encodedQueryStr = this.createSearchQueryUrl(encodedQueryStr,
					requiredResultSize, startPos, searchConditions);
			System.out.println("\t" + this.getClass().getName()
					+ "#doGetResultSet) " + (i + 1)
					+ "th (i in the program plus 1) loop out of " + repeatTimes
					+ " times.");

			// Do; Call each subclass' method.
			AbsResultSet rs_partly = this
					.doGetResultSetForSinglePageOfSearchResult(encodedQueryStr,
							analyzer);
			try {
				numofReturnedDoc = rs_partly.getEstimatedNumberOfResultsLong();
			} catch (SearchEngineQueryException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
				System.err
						.println("Error at Search Engine. Query: " + queryStr);
			}

			// Do; Only at the initial loop, recalculate the necessary number of
			// loop.
			if (i == 0 && (numofReturnedDoc < requiredResultSize)) {
				int residual, lastCycle = 0;
				residual = Integer.parseInt(Long.toString(numofReturnedDoc
						% numofResultsOnSingleResultpage));

				if (0 < residual) {
					lastCycle = 1;
				}
				repeatTimes = Integer.parseInt(Long.toString(numofReturnedDoc
						/ numofResultsOnSingleResultpage))
						+ lastCycle;
			}

			if (rs == null && rs_partly != null) {
				rs = rs_partly;
			} else if (rs_partly == null) {// 2008feb5/130s
				continue;
			} else {
				SearchResult[] results = rs_partly.getResults();
				for (int j = 0; j < results.length; j++) {
					rs.setResult(results[j]);
				}
			}

			int interval = this.getRandomInterval();
			System.out.println(" Query key: " + queryStr + "\n - loop#"
					+ (i + 1) + "\t" + super.getClass().getName()
					+ "#doGetResultSet: interval=" + interval + " sec.");
			try {
				Thread.sleep(interval * 1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		try {
			System.out.println(super.getClass().getName()
					+ "#doGetResultSet: numof result= " + rs.size());
		} catch (NullPointerException e) {
			rs = null;
		}
		return rs;
	}

	/**
	 * Uses Apache Commons HttpClient v4 internally.
	 * 
	 * @param encodedQueryStr
	 *            should be the complete form of URI that even web browser can
	 *            accept as its input on URL bar.
	 * @return Html string that corresponds to the URL given as an argument.
	 * @throws ClientProtocolException
	 * @throws IOException
	 * @since 1/3/11
	 */
	public String doGetBody(String encodedQueryStr)
			throws ClientProtocolException, IOException
	{
		System.out.println(super.getClass().getName()
				+ "#doGetBody: \n encoded query: " + encodedQueryStr);
		String responseBody = null;

		HttpClient client = new DefaultHttpClient();
		HttpGet get = new HttpGet(encodedQueryStr);
		HttpResponse response = client.execute(get);
		responseBody = EntityUtils.toString(response.getEntity());
		// responseBody = dhc.doGetBody(encodedQueryStr);

		return responseBody;
	}

	/** Preparation for ignition of API */
	// public abstract boolean init(LinkedHashMap param);
	public abstract boolean init(Properties param);

	/**
	 * TODO Needs explanation on what does "Prv" stand for.
	 * 
	 * @param encodedQueryStr
	 *            complete url
	 * @return URLs included in the search results
	 * @throws IOException
	 * @throws ClientProtocolException
	 */
	protected LinkedList<String> doGetPrv(String encodedQueryStr)
			throws ClientProtocolException, IOException
	{
		// public LinkedList doGet(String encodedQueryStr) {

		LinkedList<String> allURLs = new LinkedList<String>();
		System.out.println("***query str: " + encodedQueryStr);

		// String responseBody = getHttpResponse_HCv3(encodedQueryStr);
		String responseBody = this.doGetBody(encodedQueryStr);

		System.out.println("**body: " + responseBody);
		allURLs = this.extractDocumentURLs(responseBody);
		System.out.println("**results extracted: " + allURLs);

		// BufferedReader reader = new BufferedReader(new InputStreamReader(
		// method.getResponseBodyAsStream()));
		// System.out.println("**InputStream: " + reader);
		// StringBuffer strbuf = new StringBuffer();
		// String readLine = null;
		// int rowNum = 0;
		// while ((readLine = reader.readLine()) != null) {
		// readLine = reader.readLine();
		// // if (readLine == null) {
		// // break;
		// // }
		// strbuf.append(readLine);
		// // System.out.println("**** reading doc :" + rowNum + " line: "
		// // + readLine);
		// rowNum++;
		// }
		// String ism = strbuf.toString();
		// allResulet = this.extractDocumentUrl(ism);

		System.out.println("**results extracted: " + allURLs);

		return allURLs;
	}

	/**
	 * 
	 * @param encodedQueryStr
	 * @return
	 * @deprecated 1/3/11) Depends on HttpClient verson 3 which is obsolete.
	 */
	private String getHttpResponse_HCv3(String encodedQueryStr)
	{
		String responseBody = null;

		// 1/3/11 Toggled since this part includes components that are no longer
		// available in the current HttpClient library.
		//
		// HttpClient client = new HttpClient();
		// HttpMethod method = new GetMethod(encodedQueryStr);
		// method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
		// new DefaultHttpMethodRetryHandler(3, false));
		// try {
		// int statusCode = client.executeMethod(method);
		// System.out.println("statusCode = " + statusCode);
		//
		// Header[] headers = method.getResponseHeaders();
		// for (int i = 0; i < headers.length; i++) {
		// System.out.println("headers[" + i + "] = " + headers[i]);
		// }
		// responseBody = method.getResponseBodyAsString();
		// } catch (HttpException he) {
		// he.printStackTrace();
		// } catch (IOException ioe) {
		// ioe.printStackTrace();
		// }
		return responseBody;
	}

	protected abstract LinkedList<String> extractDocumentURLs(String body);
}